﻿using System;
using System.Collections.Generic;
using System.Text;

namespace Workflows.Components.TextMining.Clustering
{
    /// <summary>
    /// the pearson correlation coefficient is a measure of how 
    /// highly correlated two variables are. it is a value between 
    /// -1 and 1, where 1 indicates that the variables are perfectly 
    /// correlated, 0 indicates no correlation, and -1 means they are 
    /// perfectly inversely correlated
    /// </summary>
    public class PearsonCorrelationCalculator:IDistanceCalculator
    {
        #region IDistanceCalculator Members
        /// <summary>
        /// 
        /// </summary>
        /// <param name="termFreq1"></param>
        /// <param name="termFreq2"></param>
        /// <param name="contributingTerms"></param>
        /// <returns></returns>
        public double CalculateDistance(
            Dictionary<int, DocTermFreq> termFreq1,
            Dictionary<int, DocTermFreq> termFreq2,
            ref Dictionary<int, int> contributingTerms)
        {
            if (termFreq1 == null || termFreq2 == null ||
                termFreq1.Count == 0 || termFreq2.Count == 0)
                return double.PositiveInfinity;

            List<int> unionIDs = new List<int>();
            foreach(int termID1 in termFreq1.Keys)
            {
                unionIDs.Add(termID1);
            }
            foreach(int termID2 in termFreq2.Keys)
            {
                if (!unionIDs.Contains(termID2))
                    unionIDs.Add(termID2);
            }

            int sum1 = 0;
            int sumSq1 = 0;
            foreach(int termID in unionIDs)
            {
                if (termFreq1.ContainsKey(termID))
                {
                    sum1 += termFreq1[termID].Count;
                    sumSq1 +=(int) Math.Pow(termFreq1[termID].Count, 2);
                }
            }
            int sum2 = 0;
            int sumSq2 = 0;
            foreach(int termID in unionIDs)
            {
                if (termFreq2.ContainsKey(termID))
                {
                    sum2 += termFreq2[termID].Count;
                    sumSq2 += (int) Math.Pow(termFreq2[termID].Count, 2);
                }
            }
            int sumProduct = 0;
            foreach(int termID in unionIDs)
            {
                if(termFreq1.ContainsKey(termID) && termFreq2.ContainsKey(termID))
                {
                    sumProduct += termFreq1[termID].Count*termFreq2[termID].Count;
                    contributingTerms.Add(termID, Math.Min(termFreq1[termID].Count, termFreq2[termID].Count));
                }
            }
            double density =
                Math.Sqrt((sumSq1 - Math.Pow(sum1, 2)/termFreq1.Count)*(sumSq2 - Math.Pow(sum2, 2)/termFreq2.Count));
            double number = sumProduct - (sum1*sum2)/unionIDs.Count;
            double pearsonDistance = number/density;
            double docLenScale = (double)contributingTerms.Count * 2 /
                                 (termFreq1.Count + termFreq2.Count - contributingTerms.Count);
            pearsonDistance = pearsonDistance*docLenScale;
            double contributingTermScale =
                TermWeightCalculator.TaminoCommonTermScale(
                    termFreq1.Count, termFreq2.Count,
                    contributingTerms.Count);
            return pearsonDistance*contributingTermScale;
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="termFreq1"></param>
        /// <param name="termFreq2"></param>
        /// <param name="termWeights"></param>
        /// <param name="useNGramSimilarity"></param>
        /// <param name="contributingTerms"></param>
        /// <returns></returns>
        public double CalculateDistance(
            Dictionary<int, DocTermFreq> termFreq1,
            Dictionary<int, DocTermFreq> termFreq2,
            Dictionary<int, double> termWeights,
            bool useNGramSimilarity, int ngramLen,
            ref Dictionary<int, int> contributingTerms)
        {
            if (termFreq1 == null || termFreq2 == null ||
               termFreq1.Count == 0 || termFreq2.Count == 0)
                return double.PositiveInfinity;

            List<int> unionIDs = new List<int>();
            foreach (int termID1 in termFreq1.Keys)
            {
                unionIDs.Add(termID1);
            }
            foreach (int termID2 in termFreq2.Keys)
            {
                if (!unionIDs.Contains(termID2))
                    unionIDs.Add(termID2);
            }

            double sum1 = 0;
            double sumSq1 = 0;
            Dictionary<int, Dictionary<int, double>> termSimilarityMatrix =
               TermSimilarityMatrix.BuildTermDistanceMatrixUsingNgram(
                   termFreq1, termFreq2);

            foreach (int termID in unionIDs)
            {
                double t1 = 0;
                if (termFreq1.ContainsKey(termID))
                {
                    t1 = termFreq1[termID].Count*termFreq1[termID].Weight;
                }
                else if(useNGramSimilarity && ngramLen>0)
                {
                    Dictionary<int, double> similarTerms = termSimilarityMatrix[termID];
                    foreach (int similarTermID in similarTerms.Keys)
                    {
                        if (termFreq1.ContainsKey(similarTermID) &&
                            termFreq1[similarTermID].Count*termFreq1[similarTermID].Weight > t1)
                            t1 = termFreq1[similarTermID].Count * termFreq1[similarTermID].Weight;
                    }
                }
                sum1 += t1;
                double sq1 = Math.Pow(t1, 2);
                sumSq1 += sq1;
            }
            double sum2 = 0;
            double sumSq2 = 0;
            foreach (int termID in unionIDs)
            {
                double t2 = 0;
                if (termFreq2.ContainsKey(termID))
                {
                    t2 = termFreq2[termID].Count*termFreq2[termID].Weight;
                }
                else if(useNGramSimilarity && ngramLen>0)
                {
                    Dictionary<int, double> similarTerms = termSimilarityMatrix[termID];
                    foreach (int similarTermID in similarTerms.Keys)
                    {
                        if (termFreq2.ContainsKey(similarTermID) &&
                            termFreq2[similarTermID].Count * termFreq2[similarTermID].Weight > t2)
                            t2 = termFreq2[similarTermID].Count * termFreq2[similarTermID].Weight;
                    }
                }
                sum2 += t2;
                sumSq2 += Math.Pow(t2, 2);
            }
            double sumProduct = 0;
            foreach (int termID in unionIDs)
            {
                double sp = 0;
                if (termFreq1.ContainsKey(termID) && termFreq2.ContainsKey(termID))
                {
                    sp = termFreq1[termID].Count * termFreq1[termID].Count;
                    contributingTerms.Add(termID, Math.Min(termFreq1[termID].Count, termFreq2[termID].Count));
                }
                else
                {
                    if (useNGramSimilarity && ngramLen>0)
                    {
                        Dictionary<int, double> similarTerms = termSimilarityMatrix[termID];
                        foreach (int similarTermID in similarTerms.Keys)
                        {
                            if (termFreq1.ContainsKey(termID) && termFreq2.ContainsKey(similarTermID))
                            {
                                if (termFreq1[termID].Count * termFreq1[termID].Count > sp)
                                    sp = termFreq1[termID].Count * termFreq1[termID].Count;
                            }
                            else if (termFreq2.ContainsKey(termID) && termFreq1.ContainsKey(similarTermID))
                            {
                                if (termFreq2[termID].Count * termFreq2[termID].Count > sp)
                                    sp = termFreq2[termID].Count * termFreq2[termID].Count;
                            }
                        }
                    }
                }
                sumProduct += sp;
            }
            double density =
                Math.Sqrt((sumSq1 - Math.Pow(sum1, 2) / termFreq1.Count) * (sumSq2 - Math.Pow(sum2, 2) / termFreq2.Count));
            double number = sumProduct - (sum1 * sum2) / unionIDs.Count;
            double pearsonDistance = number / density;
            double docLenScale = (double)contributingTerms.Count * 2 /
                                 (termFreq1.Count + termFreq2.Count - contributingTerms.Count);
            pearsonDistance = pearsonDistance*docLenScale;

            //Dictionary<int, double> contributingTermWeights = new Dictionary<int, double>();
            //foreach (int sharedTermID in contributingTerms.Keys)
            //{
            //    if (termWeights.ContainsKey(sharedTermID))
            //    {
            //        contributingTermWeights.Add(sharedTermID, termWeights[sharedTermID]);
            //    }
            //    else
            //    {
            //        contributingTermWeights.Add(sharedTermID, 0);
            //    }
            //}
            double contributingTermScale =
                TermWeightCalculator.TaminoWeightedTermScale(
                    termFreq1, termFreq2,
                    termWeights);

            return pearsonDistance*contributingTermScale;
        }

        public double CalculateDistance(Dictionary<int, DocTermFreq> termFreq1, Dictionary<int, DocTermFreq> termFreq2,
                                        Dictionary<int, double> termWeights, Dictionary<int, string> termStemmings1,
                                        Dictionary<string, List<int>> termStemmings2,
                                        ref Dictionary<int, int> contributingTerms)
        {
            if (termFreq1 == null || termFreq2 == null ||
               termFreq1.Count == 0 || termFreq2.Count == 0)
                return double.PositiveInfinity;

            List<int> unionIDs = new List<int>();
            foreach (int termID1 in termFreq1.Keys)
            {
                unionIDs.Add(termID1);
            }
            foreach (int termID2 in termFreq2.Keys)
            {
                if (!unionIDs.Contains(termID2))
                    unionIDs.Add(termID2);
            }

            double sum1 = 0;
            double sumSq1 = 0;
            

            foreach (int termID in unionIDs)
            {
                double t1 = 0;
                if (termFreq1.ContainsKey(termID))
                {
                    t1 = termFreq1[termID].Count*termFreq1[termID].Weight;
                }
                else 
                {
                    List<int> sharedTermIDsAll = termStemmings2[termStemmings1[termID]];
                    foreach (int similarTermID in sharedTermIDsAll)
                    {
                        if (termFreq1.ContainsKey(similarTermID) &&
                            termFreq1[similarTermID].Count*termFreq1[similarTermID].Weight > t1)
                            t1 = termFreq1[similarTermID].Count * termFreq1[similarTermID].Weight;
                    }
                }
                sum1 += t1;
                double sq1 = Math.Pow(t1, 2);
                sumSq1 += sq1;
            }
            double sum2 = 0;
            double sumSq2 = 0;
            foreach (int termID in unionIDs)
            {
                double t2 = 0;
                if (termFreq2.ContainsKey(termID))
                {
                    t2 = termFreq2[termID].Count*termFreq2[termID].Weight;
                }
                else 
                {
                    List<int> sharedTermIDsAll = termStemmings2[termStemmings1[termID]];
                    foreach (int similarTermID in sharedTermIDsAll)
                    {
                        if (termFreq2.ContainsKey(similarTermID) &&
                            termFreq2[similarTermID].Count * termFreq2[similarTermID].Weight > t2)
                            t2 = termFreq2[similarTermID].Count * termFreq2[similarTermID].Weight;
                    }
                }
                sum2 += t2;
                sumSq2 += Math.Pow(t2, 2);
            }
            double sumProduct = 0;
            foreach (int termID in unionIDs)
            {
                double sp = 0;
                if (termFreq1.ContainsKey(termID) && termFreq2.ContainsKey(termID))
                {
                    sp = termFreq1[termID].Count * termFreq1[termID].Count;
                    contributingTerms.Add(termID, Math.Min(termFreq1[termID].Count, termFreq2[termID].Count));
                }
                else
                {
                    List<int> sharedTermIDsAll = termStemmings2[termStemmings1[termID]];
                    foreach (int similarTermID in sharedTermIDsAll)
                    {
                        if (termFreq1.ContainsKey(termID) && termFreq2.ContainsKey(similarTermID))
                        {
                            if (termFreq1[termID].Count*termFreq1[termID].Count > sp)
                                sp = termFreq1[termID].Count*termFreq1[termID].Count;
                        }
                        else if (termFreq2.ContainsKey(termID) && termFreq1.ContainsKey(similarTermID))
                        {
                            if (termFreq2[termID].Count*termFreq2[termID].Count > sp)
                                sp = termFreq2[termID].Count*termFreq2[termID].Count;
                        }
                    }
                }
                sumProduct += sp;
            }
            double density =
                Math.Sqrt((sumSq1 - Math.Pow(sum1, 2) / termFreq1.Count) * (sumSq2 - Math.Pow(sum2, 2) / termFreq2.Count));
            double number = sumProduct - (sum1 * sum2) / unionIDs.Count;
            double pearsonDistance = number / density;
            double docLenScale = (double)contributingTerms.Count * 2 /
                                 (termFreq1.Count + termFreq2.Count - contributingTerms.Count);
            pearsonDistance = pearsonDistance*docLenScale;

            double contributingTermScale =
                TermWeightCalculator.TaminoWeightedTermScale(
                    termFreq1, termFreq2,
                    termWeights);

            return pearsonDistance*contributingTermScale;
        }

        #endregion
    }
}
